# coding=utf-8
import traceback
import requests
from datetime import timedelta
from monthdelta import monthdelta
from pymongo import MongoClient
from simplemysql import SimpleMysql
from datetime import datetime
import Utils
from pandas import Series, DataFrame
import numpy as np
import sys

reload(sys)
sys.setdefaultencoding('utf-8')


import signal
import time

class GracefulKiller:

    def __init__(self):
        signal.signal(signal.SIGINT, self.exit_gracefully)
        signal.signal(signal.SIGTERM, self.exit_gracefully)



    def exit_gracefully(self, signum, frame):
        Utils.logger.info("python killed: signum:" + str(signum) + ", frame:" + str(frame))
        traceback.print_stack(frame)


killer = GracefulKiller()



mgClient = MongoClient('121.40.54.235', 27017,
    username='root2',
    password='Dream2015',
    authSource="stockDataStore",
    authMechanism='SCRAM-SHA-1')
db = mgClient['stockDataStore']

mysqlDB = SimpleMysql(
    host="121.40.54.235",
    db="stock-grab",
    user="root",
    passwd="Dream2015",
    charset="utf8",
    autocommit=True,
    keep_alive=True  # try and reconnect timedout mysql connections?
)

def get_stock_period_indices_from_mongo(stock_code):
    stocks = db['stock_period_indices'].find({"code": stock_code},
                              projection={"period": 1,
                                          "net_profit": 1,
                                          "net_profit_opc": 1,
                                          "net_profit_koufei": 1,
                                          "npg_rate": 1,
                                          "total_assets": 1,
                                          "net_assets": 1,
                                          "net_assets_opc": 1,
                                          "total_shares": 1}, sort=[("period", 1)]);
    all_data = list(stocks)
    all_period = map(lambda x: x['period'], all_data);
    table_data = map(lambda x: [x['net_profit'], x['net_profit_opc'],x['npg_rate'],
                                x['total_assets'], x['net_assets'], x['net_assets_opc'], x['net_profit_koufei'],
                                x['total_shares']], all_data)
    df = DataFrame(table_data,
                   columns=['net_profit','net_profit_opc','npg_rate',
                            'total_assets', 'net_assets', 'net_assets_opc', 'net_profit_koufei',
                            'total_shares'], index=all_period)

    netvalue_in_period_df = Utils.calc_netvalue_in_period(df)
    df['net_profit_opc_in_period'] = netvalue_in_period_df['net_profit_opc']
    net_profit_opc_in_period_ttm = df['net_profit_opc_in_period'].rolling(4).sum()
    df['net_profit_opc_ttm'] = net_profit_opc_in_period_ttm

    df['net_profit_koufei_in_period'] = netvalue_in_period_df['net_profit_koufei']
    net_profit_koufei_in_period_ttm = df['net_profit_koufei_in_period'].rolling(4).sum()
    df['net_profit_koufei_ttm'] = net_profit_koufei_in_period_ttm

    return df


'''
获取day所在的季度财报数据，以此来计算PB、PE等
'''
def filter_period_indices(stock_period_df, day):
    all_period = stock_period_df.index
    for i in range(len(all_period)):
        period = all_period[i]
        if i == (len(all_period) - 1):
            '''
            比如现在是2018年7月1，目前只有1季度报表，半年报在2季度结束后2个月才有结果
            今天计算的财报数据还是以1季报为准，
            所以计算以哪个财报数据为准备？年报是3个月delay，半年报是2个月delay，季报以1个月delay
            '''
            delay = 0
            if period[4:6] == '03':
                delay = 5
            if period[4:6] == '09':
                delay = 6
            if period[4:6] == '06':
                delay = 4
            if period[4:6] == '12':
                delay = 4
            next_period = (datetime.strptime(period, '%Y%m%d') + monthdelta(delay)).strftime("%Y%m%d")
            if period <= day < next_period:
                return stock_period_df.iloc[i]
        else:
            next_period = all_period[i + 1]
            if period <= day < next_period:
                return stock_period_df.iloc[i]
    return None

'''
df = get_stock_period_indices('600006')
series_data = get_period_indices(df, '20180709')
print series_data.name
print series_data.values
'''

def save_to_mongodb(period_indices):
    one = db['stock_period_indices'].find_one({'code': period_indices['code'], 'period': period_indices['period']})
    if not one:
        db['stock_period_indices'].insert_one(period_indices)
    else:
        Utils.logger.warning("period indices already exits in mongo " + period_indices['code'] + "," + period_indices['period'])


def already_in_mysql(code, day):
    max_day_cursor = mysqlDB.query("select count(day) from newlove_stock_daily_k where code = %s and day = %s", [code, day])
    count = max_day_cursor.fetchone()[0]
    if count != 0:
        return True
    else:
        return False

def already_in_mongo(code, day):
    one = db['stock_daily_k'].find_one({'code': code, 'day': day})
    if one:
        return True
    else:
        return False


def get_max_day_from_mongo(code):
    max_period_cursor = db['stock_daily_k'].aggregate([
        {"$match": {"code": code}},
        {"$group": {"_id": '$code', "max": {"$max": "$day"}}}
    ])
    result = list(max_period_cursor)
    if len(result) == 1:
        max = result[0]
        return max['max'];
    return None


def collect_daily_k(stock_code, start='19900101', end=datetime.now().strftime("%Y%m%d")):
    '''
    url:
    http://quotes.money.163.com/service/chddata.html?code=0600008&start=20000427&end=20180706&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP
    response:
    日期,股票代码,名称,收盘价,最高价,最低价,开盘价,前收盘,涨跌额,涨跌幅,换手率,成交量,成交金额,总市值,流通市值
    2018-06-22,'600987,航民股份,9.52,9.61,9.3,9.32,9.37,0.15,1.6009,0.2103,1336250,12692933.0,6048151200.0,6048151200.0
    2018-05-22,'600987,航民股份,0.0,0.0,0.0,0.0,11.29,None,None,0.0,0,0.0,7172649900.0,7172649900.0  
    '''


    #####be noted: result include both start and end day data 上面的例子包括4月27和7月6号的数据###########
    if start > end:
        print ("do not collect daily k info cause start >= T")
        return
    period_indices_df = get_stock_period_indices_from_mongo(stock_code)
    prefix = "1"##创业板和深圳中小板
    if stock_code.startswith("6"):##沪市主板
        prefix = "0"
    url = "http://quotes.money.163.com/service/chddata.html?code=" + prefix + stock_code + "&start=" + start + "&end=" + end + "&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP";
    response = requests.request("get", url);
    response_content = response.content;
    response_content = response_content.decode("GBK").encode("UTF-8")
    lines = response_content.split("\n");
    if (len(lines)) <= 1:
        Utils.logger.info ("do not find any k info cause of start, start=" + start + ", end=" + end)
        return
    lines = lines[1:]

    total_daily_k_count = len(lines)
    save_to_mongo_count = 0
    save_to_mysql_count = 0
    pe_empty_count = 0;

    ###save to disk
    #Utils.save_daily_k_data(stock_code, response_content)
    lines.reverse();
    for i in range(len(lines)):
        line = lines[i]
        if line == '':
            continue
        # 预处理
        fields = line.replace("\r", "").replace("None", '0.0').split(",")
        index = 0
        day = fields[index]
        index += 1
        code = fields[index]
        code = code.replace("'", "")
        index += 1
        name = fields[index]
        index += 1
        close = fields[index]
        index += 1
        high = fields[index]
        index += 1
        low = fields[index]
        index += 1
        open = fields[index]
        index += 1
        settlement = fields[index]
        index += 1
        pchange = fields[index]
        index += 1
        pchangepercent = fields[index]
        index += 1
        turnoverrate = fields[index]
        index += 1
        volume = fields[index]
        index += 1
        amount = fields[index]
        index += 1
        mktcap = fields[index]
        index += 1
        nmc = fields[index]
        day = day.replace("-", '')
        if index != len(fields) - 1:
            raise Exception("k线信息长度不一致" + index + "e" + (len(fields) - 1))
        #print(day + "/" + code + "/" + mktcap)
        if close == '0.0' or low == '0.0' or high == '0.0' or  nmc == '' or mktcap == '':
            continue;
        totalShares = long(float(mktcap) / float(close));
        period_indices_series_data_ = filter_period_indices(period_indices_df, day)
        if period_indices_series_data_ is None:
            Utils.logger.info("period indices data is empty")
        pe_ttm = float(0)
        pe_koufei_ttm = float(0)
        peg = float(0)
        pb = float(0)
        net_assets = 0
        net_profit = float(0)
        net_profit_ttm = float(0)
        net_profit_koufei_ttm = float(0)
        total_assets = float(0)

        if period_indices_series_data_ is not None:
            net_profit_ttm = period_indices_series_data_["net_profit_opc_ttm"]
            net_profit_koufei_ttm = period_indices_series_data_["net_profit_koufei_ttm"]
            npg_rate = period_indices_series_data_["npg_rate"]

            net_assets = period_indices_series_data_["net_assets_opc"]
            net_profit = period_indices_series_data_["net_profit_opc"]
            total_assets = period_indices_series_data_["total_assets"]

            if net_profit_ttm is not None and net_profit_ttm !=0:
                pe_ttm = float(mktcap)/net_profit_ttm
            if net_profit_koufei_ttm is not None and net_profit_koufei_ttm !=0:
                pe_koufei_ttm = float(mktcap)/net_profit_koufei_ttm
            if npg_rate is not None and npg_rate !=0:
                peg = float(pe_ttm/(npg_rate*100))
            if net_assets is not None and net_assets != 0:
                pb = float(mktcap)/net_assets
        if np.isnan(pe_ttm):
            pe_ttm = float(0)
        if np.isnan(pe_koufei_ttm):
            pe_koufei_ttm = float(0)
        if np.isnan(peg):
            peg = float(0)
        if np.isnan(pb):
            pb = float(0)

        if np.isnan(net_profit_ttm):
            net_profit_ttm = float(0)
        if np.isnan(net_profit_koufei_ttm):
            net_profit_koufei_ttm = float(0)
        if np.isnan(total_assets):
            total_assets = float(0)
        if np.isnan(net_assets):
            net_assets = float(0)

        record = {'day': day, 'code': code, 'name': name, 'close': float(close), 'high': float(high), 'low': float(low),
                        'open': float(open), 'settlement': float(settlement), 'pchange': float(pchange), 'pchange_percent': float(pchangepercent),
                        'turnover_rate': float(turnoverrate), 'volume': float(volume), 'amount': float(amount), 'mktcap': float(mktcap), 'nmc': float(nmc),
                        'pb': float(pb), 'pe_ttm': float(pe_ttm), 'pe_koufei_ttm': pe_koufei_ttm, 'peg': float(peg), 'total_shares': float(totalShares),
                        'net_assets': float(net_assets), 'net_profit': float(net_profit), 'net_profit_ttm': float(net_profit_ttm),
                        'net_profit_koufei_ttm': float(net_profit_koufei_ttm), 'total_assets': float(total_assets),
                        'createTime': datetime.now(),'modifyTime': datetime.now()}


        if not already_in_mongo(code, day):
            mongoRecord = record.copy()  ##为什么要copy？存储到mongo后record 多了一个_id 属性导致mysql存储失败
            #Utils.logger.info("save record to mongo of stock " + mongoRecord['code'] + " " + mongoRecord['day'])
            db['stock_daily_k'].insert_one(mongoRecord)
            save_to_mongo_count = save_to_mongo_count + 1

        #if not already_in_mysql(code, day):
            #mysqlDB.insert("newlove_stock_daily_k", record)
            #save_to_mysql_count = save_to_mysql_count + 1

        if (pe_ttm == float(0)):
            pe_empty_count = pe_empty_count + 1

    Utils.logger.info("count of empty pe_ttm is" +str(pe_empty_count) + " and count of record to save to mysql/mongo count" + str(save_to_mysql_count) + "/" + str(save_to_mongo_count))
    #mysqlDB.commit()


def collect(stocks):
    for stock in list(stocks):
        if Utils.bypass_stock(stock['code']):
            Utils.logger.info(stock['code'] + " bypassed ")
            continue
        try:
            Utils.logger.info("--->collect daily k of " + stock["code"] + " "+ stock["name"])
            code = stock['code']
            start_day = get_max_day_from_mongo(code)
            if start_day is None:
                start_day = '19980101'
            Utils.logger.info("--->latest day of " + stock["code"] + " " + stock["name"] + start_day)
            collect_daily_k(code, start_day)
            Utils.logger.info("--->success collect daily k of " + stock["code"] + " "+ stock["name"] + "\n")
        #except Exception as es:
        except:
            traceback.print_exc()
            ##mysqlDB.end()
            Utils.logger.error("fail to collect daily k of " + stock['code'] + "\n")

def entrance():
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^00"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    collect(stocks)
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^30"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    collect(stocks)
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^60"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    collect(stocks)

entrance()
#stocks=[{"code":'600009', 'name':'上海机场'}]
#collect(stocks)
Utils.logger.info("CalcStockDailyK.py exit")
